import scrapy
from faker import Faker
from meishi.items import MztrItem
from meishi.items import StrItem
import random
import time
import json
import sys
import requests as req
from lxml import etree
import math

class MztrSpider(scrapy.Spider):
	name = 'mztr'
	allowed_domains = ['meituan.com']
	start_urls = ['https://www.meituan.com/changecity/']
	header={
	'User-Agent':Faker(locale='zh_CN').chrome()
	}

	def parse(self,response):
		city_list = response.xpath("//div[@class='city-area']/span/a")
		pds = city_list[655]
		# city_name = pds.xpath("./text()").extract()[0]
		pds_url = "https:"+pds.xpath("./@href").extract()[0]+"/meishi/"
		yield scrapy.Request(pds_url,headers=self.header,callback=self.parse_two,dont_filter=True,meta={'pds_url':pds_url})
		# html = req.get(pds_url,headers=self.header).content.decode("utf8")

	def parse_two(self,response):
		# jxxi = etree.HTML(html)
		pds_url = response.meta['pds_url']
		jx = response.xpath("//script/text()").extract()[5]
		hide_str = jx+""
		hide_dict = hide_str[19:-1]
		hide_json = json.loads(hide_dict)
		all_shop_num = hide_json['poiLists']['totalCounts']
		all_page_num = int(math.ceil(float(all_shop_num)/15))		#向上取整

		for page in range(1,all_page_num):
			url = pds_url+"pn"+str(page)+"/"
			time.sleep(random.uniform(1,3))
			yield scrapy.Request(url,headers=self.header,callback=self.parse_three,dont_filter=True)
			# html = req.get(url,headers=self.header).content.decode("utf8")

	def parse_three(self,response):
		html = response.xpath("//script/text()").extract()[5]
		hide_str = html+""
		item = StrItem()
		item['str'] = hide_str
		yield item



















	# 获取单个菜单的响应,请求每页
	def parse_four(self,response):

		city_name = response.meta['city_name']
		url = response.meta['url']
		shop_list = response.xpath("//*[@id='app']/section/div/div[2]/div[2]/div[1]/ul/li")

		# 对单页中每个餐厅遍历
		for shop in shop_list:
			item = MztrItem()
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a
			item['shopId'] = shop.xpath("./div[2]/a/@href").extract()[0]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[1]/a/div[2]/img
			item['frontimg'] = shop.xpath("./div[1]/a/div[2]/img/@src").extract()[0]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/h4
			item['shopName'] = shop.xpath("./div[@class='info']/a/h4/text()").extract()[0]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/div/p/text()[1]
			item['avgScore'] = shop.xpath("./div[@class='info']/a/div/p/text()").extract()[1]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/div/p/span/text()[1]
			item['allCommentNum'] = shop.xpath("./div[@class='info']/a/div/p/span/text()").extract()[1]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/p/text()
			item['address'] = shop.xpath("./div[@class='info']/a/p/text()").extract()[0]
			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/p/span/text()[2]
			item['avgPrice'] = shop.xpath("./div[2]/a/p/span/text()").extract()[2]
			item['cityName'] = city_name
			# time.sleep(random.uniform(1,3))
			yield item
		self.page+=1
		time.sleep(random.uniform(1,3))
		yield scrapy.Request(url+"pn"+str(self.page)+"/",
			callback=self.parse_three,
			headers={'User-Agent':Faker(locale='zh_CN').chrome(),
					'Referer':'https://pds.meituan.com/meishi/'},
			dont_filter=True,
			meta={'url':url,'city_name':city_name})



































# class MztrSpider(scrapy.Spider):
# 	name = 'mztr'
# 	allowed_domains = ['meituan.com']
# 	start_urls = ['https://www.meituan.com/changecity/']
# 	header01={
# 	'User-Agent':Faker(locale='zh_CN').chrome(),
# 	'Referer':'https://pds.meituan.com/'
# 	}

# 	page = 1

# 	# 获取平顶山的链接，请求美食页面
# 	def parse(self,response):
# 		city_list = response.xpath("//div[@class='city-area']/span/a")
# 		pds = city_list[655]
# 		city_name = pds.xpath("./text()").extract()[0]
# 		pds_url = "https:"+pds.xpath("./@href").extract()[0]+"/meishi/"
# 		time.sleep(random.uniform(1,3))
# 		yield scrapy.Request(pds_url,callback=self.parse_two,dont_filter=True,headers=self.header01,meta={"city_name":city_name,'pds_url':pds_url})

# 	# 获取美食链接,请求菜单页面
# 	def parse_two(self,response):
# 		city_name = response.meta["city_name"]
# 		two_list = response.xpath("//*[@id='app']/section/div/div[2]/div[1]/div/div[1]/ul/li/a")
# 		two_list.pop(0)
# 		# 遍历每种菜单
# 		for two in two_list:
# 			two_name = two.xpath("./text()").extract()[0]
# 			url = two.xpath("./@href").extract()[0].split(":")
# 			two_url = url[0]+"s:"+url[1]

# 			time.sleep(random.uniform(1,3))
# 			yield scrapy.Request(two_url,
# 				callback=self.parse_three,
# 				headers={'User-Agent':Faker(locale='zh_CN').chrome(),
# 						'Referer':'https://pds.meituan.com/meishi/'},
# 				dont_filter=True,
# 				meta={"two_name":two_name,"city_name":city_name,'url':two_url})

# 	# 获取单个菜单的响应,请求每页
# 	def parse_three(self,response):
# 		if response.status==404:
# 			self.page = 1
# 			return
# 		city_name = response.meta['city_name']
# 		two_name = response.meta['two_name']
# 		url = response.meta['url']

# 		shop_list = response.xpath("//*[@id='app']/section/div/div[2]/div[2]/div[1]/ul/li")

# 		# test11 = response.xpath("//*[@id='app']/section/div/div[2]/div[2]/div[2]/ul/li")
# 		# print(response)
# 		# print(test11)
# 		# print(test11[-2].xpath("./span/text()").extract()[0])
# 		# print(type(test11))
# 		# sys.exit(0)
# 		# all_page = int(response.xpath("//*[@id='app']/section/div/div[2]/div[2]/div[2]/ul/li/span/text()").extract()[-1])

# 		# 对单页中每个餐厅遍历
# 		for shop in shop_list:
# 			item = MztrItem()

# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a
# 			item['shopId'] = shop.xpath("./div[2]/a/@href").extract()[0]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[1]/a/div[2]/img
# 			item['frontimg'] = shop.xpath("./div[1]/a/div[2]/img/@src").extract()[0]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/h4
# 			item['shopName'] = shop.xpath("./div[@class='info']/a/h4/text()").extract()[0]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/div/p/text()[1]
# 			item['avgScore'] = shop.xpath("./div[@class='info']/a/div/p/text()").extract()[1]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/div/p/span/text()[1]
# 			item['allCommentNum'] = shop.xpath("./div[@class='info']/a/div/p/span/text()").extract()[1]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/p/text()
# 			item['address'] = shop.xpath("./div[@class='info']/a/p/text()").extract()[0]
# 			# //*[@id="app"]/section/div/div[2]/div[2]/div[1]/ul/li[1]/div[2]/a/p/span/text()[2]
# 			item['avgPrice'] = shop.xpath("./div[2]/a/p/span/text()").extract()[2]
# 			item['cityName'] = city_name
# 			item['shopstStyle'] = two_name
# 			# time.sleep(random.uniform(1,3))
# 			yield item
# 		self.page+=1
# 		time.sleep(random.uniform(1,3))
# 		yield scrapy.Request(url+"pn"+str(self.page),
# 			callback=self.parse_three,
# 			headers={'User-Agent':Faker(locale='zh_CN').chrome()},
# 			dont_filter=True,
# 			meta={'url':url,'city_name':city_name,'two_name':two_name})


